import requests
from pyquery import PyQuery as pq
import LConf
import EasySqlite
import random


def getUA(jsonPath):
    '''随机抽取json文件中的一个UA
    '''
    res = LConf.rJson(jsonPath)
    r = random.randint(0, len(res) - 1)
    return {'user-agent': res[r]}


def getTxt(url):
    '''从详情页获取小说内容
    '''
    h = getUA('UA.json')
    conf = LConf.iniConf('adv_spyTXT.ini')
    timeout = int(conf['timeout'])
    r = False
    try:
        r = requests.get(url, timeout=timeout, headers=h)
        r.encoding = 'utf-8'
        html = r.text
    except Exception as e:
        if r:
            r.close()
        return False, False
    doc = pq(html)
    doc = doc('.box-main-content')
    title = doc('h1').text()
    con = doc('.xs-details-content').text()
    return con, title


def getUrlList(url):
    '''从列表页获取详情页面的连接池
    '''
    conf = LConf.iniConf('adv_spy.ini')
    timeout = int(conf['timeout'])
    res = []
    r = False
    try:
        r = requests.get(url, timeout=timeout)
        r.encoding = 'utf-8'
        html = r.text
    except Exception as e:
        if r:
            r.close()
        return False, False, False
    doc = pq(html)
    pageDom = doc('#long-page')  # 翻页信息
    pageDom = pageDom('a.next')
    np = pageDom.eq(0).attr.href
    lp = pageDom.eq(1).attr.href
    doc = doc('.box-main-content')
    cDom = doc('.box-title')
    classes = cDom('h3').text()
    ulDom = doc('ul').eq(1)
    liDom = ulDom('li')
    res = []
    for liItem in liDom.items():
        aitem = liItem('a')
        res.append(aitem.attr.href)
    return res, np, lp


def absUrl(domain, url):
    '''获取绝对路径
    '''
    if 'http' in url:
        return url
    else:
        return domain + url


def pageParam(url):
    '''获取分页参数 URL路由的最后一段
    '''
    tmp = url.split('/')
    lPart = tmp[-1]
    pathUrl = url.replace(lPart, '')
    lPartArr = tmp[-1].split('.')
    param = lPartArr[0]
    suffix = lPartArr[-1]
    return pathUrl, param, suffix


def step_1(conf):
    logPath = conf['logPath']
    db = EasySqlite.EasySqlite('./spy_txt.db')  # 连接数据库
    if isinstance(conf['listPath'], str):
        conf['listPath'] = (conf['listPage'],)

    for typeUrl in conf['listPath']:
        sPurl = absUrl(conf['domain'], typeUrl)
        lurl, nPurl, lPurl = getUrlList(sPurl)
        if lurl is False:
            continue
        for one in lurl:
            one = absUrl(conf['domain'], one)
            db.table('task').insert({'url': one, 'listPath': sPurl, 'level': 1})
        db.commit()
        pargeInfo = pageParam(nPurl)
        st = int(pargeInfo[1])
        ed = int(pageParam(lPurl)[1])
        for i in range(st, ed + 1):
            jointUrl = sPurl + '/' + str(i) + '.' + pargeInfo[2]
            jointUrl = absUrl(conf['domain'], jointUrl)
            db.table('task').insert({'url': jointUrl, 'level': 0})
        db.commit()
        printStr = "添加栏目：%s（%d) 添加页数：%d" % (typeUrl, ed - st + 1, len(lurl))
        LConf.Wlog(logPath, printStr, 'a')
        print(printStr)


def step_2(conf):
    logPath = conf['logPath']
    RETSQL = 'reTime<' + conf['reTime']
    db = EasySqlite.EasySqlite('./spy_txt.db')
    while True:
        count = db.table('task').where(['status=0']).count()
        item = db.table('task').where(['status=0', RETSQL]).field(['id', 'url', 'level', 'listPath', 'reTime']).find()
        if not item:
            break
        if int(item[2]) == 1:  # level=1 获取详情页的下载连接
            content, title = getTxt(item[1])
            if not content:
                db.table('task').where("id=%d" % item[0]).update({'reTime': item[4] + 1})
                printStr = "id:%d 错误%d次 " % (item[0], item[4] + 1)
                LConf.Wlog(logPath, printStr, 'a')
                print(printStr)
                continue
            db.table('res').insert({'url': item[1], 'content': content, 'listPath': item[3], 'title': title})
            printStr = "任务池剩余：%d 当前任务（id=%d）：%s" % (count, item[0], title)
            LConf.Wlog(logPath, printStr, 'a')
            print(printStr)
        elif int(item[2]) == 0:  # level=0 获取列表页所有详情页地址
            res = getUrlList(item[1])
            listPath = pageParam(item[1])[0]
            if not res[0]:
                db.table('task').where("id=%d" % item[0]).update({'reTime': item[4] + 1})
                LConf.Wlog(logPath, printStr, 'a')
                print(printStr)
                continue
            for one in res[0]:
                one = absUrl(conf['domain'], one)
                db.table('task').insert({'url': one, 'listPath': listPath, 'level': 1})
            printStr = "任务池 +%d = %d" % (len(res[0]), count + len(res[0]) - 1)
            LConf.Wlog(logPath, printStr, 'a')
            print(printStr)
        else:
            continue
        db.table('task').where("id=%d" % item[0]).update({'status': 1})
        db.commit()


def initSqlite(sqlPath):
    db = EasySqlite.EasySqlite('./spy_txt.db')
    res = db.execSqlScript(sqlPath)
    if res:
        db.commit()
        return True
    else:
        print(db.errorMsg)
        return False


def main():
    conf = LConf.iniConf('./adv_spyTXT.ini')
    if int(conf['init']) == 1:
        if initSqlite(conf['initSql']) is False:
            return 0
        step_1(conf)
    elif int(conf['init']) == 2:
        step_2(conf)


if __name__ == '__main__':
    main()
    # res = getTxt('https://www.106fu.com/htm/novel1/112188.htm')
    # res = getUrlList('https://www.106fu.com/htm/novellist1/')
    # res = initSqlite('./adv_txt.sql')
    # conf = LConf.iniConf('./adv_spyTXT.ini')
    # step_1(conf)
    # print(res)
